{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from math import log\n",
    "from functools import reduce\n",
    "%matplotlib inline\n",
    "\n",
    "import pickle"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "fsfc = open(\"fcount.pkl\",'rb')\n",
    "fset = pickle.load(fsfc)\n",
    "\n",
    "fsipd = open(\"ip_dict.pkl\",'rb')\n",
    "[dic,dc] = pickle.load(fsipd)\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "l = [\"\"] * len(dic)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "for x in dic:\n",
    "    l[dic[x]] = x\n",
    "    "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "d = {}\n",
    "site_null = \"85f751fd\"\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\"train_sub_count.csv\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(\"test_sub_count.csv\")    "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "count = 0"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "def prep(df):\n",
    "    global count,d\n",
    "    count +=1\n",
    "    if count%1000000 == 0:\n",
    "        print(count)\n",
    "    ip = df['device_ip']\n",
    "    if \"device_ip_\" + ip in fset:\n",
    "        w = df[\"site_category\"]\n",
    "        if df['site_id'] == site_null:\n",
    "            w = df['app_category']\n",
    "        if ip not in d:\n",
    "            d[ip] = [0.] *len(dic)\n",
    "        d[ip][dic[w]] += 1\n",
    "    "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "1000000\n",
      "2000000\n",
      "3000000\n",
      "4000000\n",
      "5000000\n",
      "6000000\n",
      "7000000\n",
      "8000000\n",
      "9000000\n",
      "10000000\n"
     ],
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "0          None\n1          None\n2          None\n3          None\n4          None\n           ... \n9999995    None\n9999996    None\n9999997    None\n9999998    None\n9999999    None\nLength: 10000000, dtype: object"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 10
    }
   ],
   "source": [
    "train_df.apply(prep,axis=1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "11000000\n",
      "12000000\n",
      "13000000\n",
      "14000000\n"
     ],
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "0          None\n1          None\n2          None\n3          None\n4          None\n           ... \n4577459    None\n4577460    None\n4577461    None\n4577462    None\n4577463    None\nLength: 4577464, dtype: object"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 11
    }
   ],
   "source": [
    "test_df.apply(prep,axis=1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [],
   "source": [
    "ll = float(len(d)) # ip 个数\n",
    "for k in d:\n",
    "    s = reduce(lambda x,y:x + y,d[k]) # d[k] 求和\n",
    "    for i in range(len(d[k])):\n",
    "\n",
    "        d[k][i] = d[k][i] / s * log(ll / dc[l[i]])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "pickle.dump(d, open(\"ip_mat.pkl\", 'wb'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "#d_tdf = pd.DataFrame.from_dict(d_test)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "#columns_org = d_tdf.columns"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "# transform counts to TFIDF features\n",
    "# from sklearn.feature_extraction.text import TfidfTransformer\n",
    "# tfidf = TfidfTransformer()\n",
    "# \n",
    "# #输出稀疏矩阵\n",
    "# X_train_tfidf = tfidf.fit_transform(d_tdf).toarray()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# X_train_tfidf = pd.DataFrame(columns = columns_org, data = X_train_tfidf)\n",
    "\n",
    "# X_train_tfidf.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}